
#include "RBM.h"

#define MAX_GRADIENT					100000.0f

#define COUNT_TIME						1

// [*] nw_rbm::CWeightItem

nw_rbm::CWeightItem::CWeightItem()
{
	fValueFixed = false;
	ClearAll();
}

void nw_rbm::CWeightItem::ClearAll()
{
	ClearValue();
	ClearConDiv();
	ClearSpaCod();
}

void nw_rbm::CWeightItem::ClearValue()
{
	Value = 0.0f;
}

void nw_rbm::CWeightItem::ClearConDiv()
{
	ConDiv_NextIncrease = 0.0f;
	ConDiv_LastIncrease = 0.0f;
	ConDiv_CumulativeWeight = 0.0f;
}

void nw_rbm::CWeightItem::ClearSpaCod()
{
	SpaCod_CumulativeError = 0.0f;
	SpaCod_CumulativeDerivative = 0.0f;
	SpaCod_CumulativeWeight = 0.0f;
}

// [*] nw_rbm::CUnitBasic

nw_rbm::CUnitBasic::CUnitBasic()
{
	myInit(false, 1.0f, -1);
}

void nw_rbm::CUnitBasic::myInit(
	bool fIsRealValued,	RBM_FLOAT Sigma, int BiasIndex)
{
	// Yuan Lin
	if (Sigma < MINIMUM_DOUBLE_EPS_C)
	{
		fIsRealValued = false;
		Sigma = 1.0f;
	}

	// Yuan Lin
	if (!fIsRealValued) Sigma = 1.0f;

	this->fIsRealValued = fIsRealValued;
	this->Sigma = Sigma;
	this->BiasIndex = BiasIndex;
	ClearValue();
}

void nw_rbm::CUnitBasic::ClearValue()
{
	Energy = 0.0f;
	Expectation = 0.5f;
	State = 0.0f;
}

RBM_FLOAT nw_rbm::CUnitBasic::InferenceEnergy(
	const CConnectionManager* qConnectionManager,
	int SelfHandle,
	const CWeightItem* PtrTargetBiasWeightItem,
	const CUnitBasic* PtrSourceUnit,
	const CWeightItem* PtrEdgeWeightItem)
{
	Energy = (BiasIndex<0 ? 0.0f : PtrTargetBiasWeightItem[BiasIndex].Value);

	int i;
	RBM_FLOAT tu, tw;
	const int* qUI;
	const int* qWI;
	int nU;
	CSimpleTypeArray<int> vUI;
	CSimpleTypeArray<int> vWI;
	nU = CConnectionManager::GetUnitConnectionList(
		qConnectionManager, SelfHandle, vUI, vWI);
	qUI = vUI.pElement; qWI = vWI.pElement;
	for ( i = 0; i < nU; ++i, ++qUI, ++qWI )
	{
		tu = PtrSourceUnit[*qUI].State;
		tw = PtrEdgeWeightItem[*qWI].Value;
		Energy += tu * tw;
	}
	return Energy;
}

void nw_rbm::CUnitBasic::HelpOtherUnitsInferenceEnergy(
	const CConnectionManager* qConnectionManager,
	int SelfHandle,
	CUnitBasic* PtrTargetUnit,
	const CWeightItem* PtrEdgeWeightItem )
{
	if (State>-EPS && State<+EPS) return;
	int i;
	RBM_FLOAT tw;
	const int* qUI;
	const int* qWI;
	int nU;
	CSimpleTypeArray<int> vUI;
	CSimpleTypeArray<int> vWI;
	nU = CConnectionManager::GetUnitConnectionList(
		qConnectionManager, SelfHandle, vUI, vWI);
	qUI = vUI.pElement; qWI = vWI.pElement;
	for ( i = 0; i < nU; ++i, ++qUI, ++qWI )
	{
		tw = PtrEdgeWeightItem[*qWI].Value;
		PtrTargetUnit[*qUI].Energy += tw * State;
	}
}

RBM_FLOAT nw_rbm::CUnitBasic::ComputeExpectation()
{
	if (fIsRealValued)
		Expectation = 0.5f + Energy * Sigma * Sigma;
	else
		Expectation = CMyMath::logistic(Energy);
	return Expectation;
}

RBM_FLOAT nw_rbm::CUnitBasic::SampleState_ByEnergy( RBM_FLOAT Energy )
{
	this->Energy = Energy;
	ComputeExpectation();
	SampleState();
	return State;
}

RBM_FLOAT nw_rbm::CUnitBasic::SampleState_ByExpectation( RBM_FLOAT Expectation )
{
	this->Expectation = Expectation;
	SampleState();
	return State;
}

RBM_FLOAT nw_rbm::CUnitBasic::SampleState()
{
	if (fIsRealValued)
		State = (RBM_FLOAT)CMyRandom::Gaussian1D(Expectation, Sigma);
	else
		State = (RBM_FLOAT)(CMyRandom::NextDouble()<Expectation ? 1.0f : 0.0f);
	return State;
}

// [*] nw_rbm::C_RBM_InferenceHelper

nw_rbm::C_RBM_InferenceHelper::C_RBM_InferenceHelper()
{
}

nw_rbm::C_RBM_InferenceHelper::~C_RBM_InferenceHelper()
{
}

void nw_rbm::C_RBM_InferenceHelper::InferenceEnergy_of_Units_Visible(CRestrictedBoltzmannMachine* pRBM)
{
	if (pRBM == NULL) return;

	int j;
	pRBM->SetBiasOnlyEnergy_of_Units_Visible();
	for ( j = 0; j < pRBM->AmountUnit_Hidden; ++j )
	{
		pRBM->ListUnit_Hidden.pElement[j].HelpOtherUnitsInferenceEnergy(
			pRBM->pConnectionManager_Hidden, j,
			pRBM->ListUnit_Visible.pElement,
			pRBM->ListWeightItem_Edge.pElement);
	}
}

void nw_rbm::C_RBM_InferenceHelper::InferenceEnergy_of_Units_Hidden(CRestrictedBoltzmannMachine* pRBM)
{
	if (pRBM == NULL) return;

	int i;
	pRBM->SetBiasOnlyEnergy_of_Units_Hidden();
	for ( i = 0; i < pRBM->AmountUnit_Visible; ++i )
	{
		pRBM->ListUnit_Visible.pElement[i].HelpOtherUnitsInferenceEnergy(
			pRBM->pConnectionManager_Visible, i,
			pRBM->ListUnit_Hidden.pElement,
			pRBM->ListWeightItem_Edge.pElement);
	}
}

RBM_FLOAT nw_rbm::C_RBM_InferenceHelper::ComputeEdgeGradient_ConDiv(
	CRestrictedBoltzmannMachine* pRBM,
	const RBM_FLOAT* qVisibleState_Data,
	const RBM_FLOAT* qHiddenExpectation_Data,
	const RBM_FLOAT* qVisibleState_Model,
	const RBM_FLOAT* qHiddenExpectation_Model,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment )
{
	RBM_FLOAT VisibleError = 0.0f;

	if (pRBM==NULL
		|| qVisibleState_Data==NULL
		|| qHiddenExpectation_Data==NULL
		|| qVisibleState_Model==NULL
		|| qHiddenExpectation_Model==NULL
		|| abs(ExampleWeight)<EPS
		)
		return VisibleError;

	int i_V, ee, j_H, o;
	int nU;
	CSimpleTypeArray<int> vUI;
	CSimpleTypeArray<int> vWI;
	const int* qUI;
	const int* qWI;
	RBM_FLOAT dw;
	CWeightItem* pWI;

	GradientAmendment *= ExampleWeight;

	pWI = pRBM->ListWeightItem_Visible.pElement;
	for ( i_V = 0; i_V < pRBM->AmountUnit_Visible; ++i_V )
	{
		const CUnitBasic& tVi = pRBM->ListUnit_Visible.pElement[i_V];
		o = tVi.BiasIndex;
		dw = qVisibleState_Data[i_V] - qVisibleState_Model[i_V];
		if (o >= 0)
		{
			pWI[o].ConDiv_NextIncrease += dw * GradientAmendment;
			pWI[o].ConDiv_CumulativeWeight += ExampleWeight;
		}
		VisibleError += dw * dw;
	}
	VisibleError = sqrt(VisibleError / pRBM->AmountUnit_Visible);
	VisibleError *= ExampleWeight;

	pWI = pRBM->ListWeightItem_Hidden.pElement;
	for ( j_H = 0; j_H < pRBM->AmountUnit_Hidden; ++j_H )
	{
		const CUnitBasic& tHj = pRBM->ListUnit_Hidden.pElement[j_H];
		o = tHj.BiasIndex;
		dw = qHiddenExpectation_Data[j_H] - qHiddenExpectation_Model[j_H];
		if (o >= 0)
		{
			pWI[o].ConDiv_NextIncrease += dw * GradientAmendment;
			pWI[o].ConDiv_CumulativeWeight += ExampleWeight;
		}
	}

	pWI = pRBM->ListWeightItem_Edge.pElement;
	for ( i_V = 0; i_V < pRBM->AmountUnit_Visible; ++i_V )
	{
		const CUnitBasic& tVi = pRBM->ListUnit_Visible.pElement[i_V];
		//if (abs(qVisibleState_Data[i_V])+abs(qVisibleState_Model[i_V]) < EPS+EPS)
		//	continue;
		nU = CConnectionManager::GetUnitConnectionList(
			pRBM->pConnectionManager_Visible, i_V, vUI, vWI);
		qUI = vUI.pElement; qWI = vWI.pElement;
		for ( ee = 0; ee < nU; ++ee )
		{
			j_H = qUI[ee]; o = qWI[ee];
			dw = qVisibleState_Data[i_V]*qHiddenExpectation_Data[j_H]
				- qVisibleState_Model[i_V]*qHiddenExpectation_Model[j_H];
			pWI[o].ConDiv_NextIncrease += dw * GradientAmendment;
			pWI[o].ConDiv_CumulativeWeight += ExampleWeight;
		}
	}

	return VisibleError;
}

// [*] nw_rbm::CRestrictedBoltzmannMachine

nw_rbm::CRestrictedBoltzmannMachine::CRestrictedBoltzmannMachine()
{
	pConnectionManager_Visible = NULL;
	pConnectionManager_Hidden = NULL;
	Clear_WeiUniTra_Settings();
	ClearTimers();
}

nw_rbm::CRestrictedBoltzmannMachine::CRestrictedBoltzmannMachine( const CRestrictedBoltzmannMachine& Another )
{
	pConnectionManager_Visible = CConnectionManager::myClone(Another.pConnectionManager_Visible);
	pConnectionManager_Hidden = CConnectionManager::myClone(Another.pConnectionManager_Hidden);
}

nw_rbm::CRestrictedBoltzmannMachine& nw_rbm::CRestrictedBoltzmannMachine::operator=( const CRestrictedBoltzmannMachine& Another )
{
	AmountUnit_Visible = Another.AmountUnit_Visible;
	ListUnit_Visible = Another.ListUnit_Visible;
	AmountUnit_Hidden = Another.AmountUnit_Hidden;
	ListUnit_Hidden = Another.ListUnit_Hidden;
	AmountWeightItem_Edge = Another.AmountWeightItem_Edge;
	AmountWeightItem_Visible = Another.AmountWeightItem_Visible;
	AmountWeightItem_Hidden = Another.AmountWeightItem_Hidden;
	ListWeightItem_Edge = Another.ListWeightItem_Edge;
	ListWeightItem_Visible = Another.ListWeightItem_Visible;
	ListWeightItem_Hidden = Another.ListWeightItem_Hidden;
	PrintEdgeWeightAmountPerLine = Another.PrintEdgeWeightAmountPerLine;
	PrintEdgeWeightAmountPerKind = Another.PrintEdgeWeightAmountPerKind;
	Training_CumulativeVisibleError = Another.Training_CumulativeVisibleError;
	Training_CumulativeHiddenPenalty = Another.Training_CumulativeHiddenPenalty;
	Training_CumulativeExampleWeight_VisibleError = Another.Training_CumulativeExampleWeight_VisibleError;
	Training_CumulativeExampleWeight_ConDiv = Another.Training_CumulativeExampleWeight_ConDiv;
	Training_CumulativeExampleWeight_SpaCod = Another.Training_CumulativeExampleWeight_SpaCod;
	Training_DeltaWeight_ConDiv = Another.Training_DeltaWeight_ConDiv;
	Training_DeltaWeight_SpaCod = Another.Training_DeltaWeight_SpaCod;
	ClearTimers();
	pConnectionManager_Visible = CConnectionManager::myClone(Another.pConnectionManager_Visible);
	pConnectionManager_Hidden = CConnectionManager::myClone(Another.pConnectionManager_Hidden);
	return *this;
}

nw_rbm::CRestrictedBoltzmannMachine::~CRestrictedBoltzmannMachine()
{
	CDataStructureTools::MyRelease_Single(pConnectionManager_Visible);
	CDataStructureTools::MyRelease_Single(pConnectionManager_Hidden);
}

void nw_rbm::CRestrictedBoltzmannMachine::SetConnectionManagers(
	CConnectionManager* pNewConnectionManager_Visible,
	CConnectionManager* pNewConnectionManager_Hidden )
{
	if (pConnectionManager_Visible != pNewConnectionManager_Visible)
	{
		CDataStructureTools::MyRelease_Single(pConnectionManager_Visible);
		pConnectionManager_Visible = pNewConnectionManager_Visible;
	}
	if (pConnectionManager_Hidden != pNewConnectionManager_Hidden)
	{
		CDataStructureTools::MyRelease_Single(pConnectionManager_Hidden);
		pConnectionManager_Hidden = pNewConnectionManager_Hidden;
	}
}

void nw_rbm::CRestrictedBoltzmannMachine::Clear_WeiUniTra_Settings()
{
	ClearWeightSettings();
	ClearUnitSettings();
	ClearTrainingInfo();
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearWeightSettings()
{
	AmountWeightItem_Edge = 0;
	ListWeightItem_Edge.Clear();

	AmountWeightItem_Visible = 0;
	ListWeightItem_Visible.Clear();

	AmountWeightItem_Hidden = 0;
	ListWeightItem_Hidden.Clear();
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearUnitSettings()
{
	AmountUnit_Visible = 0;
	ListUnit_Visible.Clear();
	AmountUnit_Hidden = 0;
	ListUnit_Hidden.Clear();
	PrintEdgeWeightAmountPerLine = 1;
	PrintEdgeWeightAmountPerKind = 1;
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearTrainingInfo()
{
	Training_CumulativeVisibleError = 0.0f;
	Training_CumulativeHiddenPenalty = 0.0f;
	Training_DeltaWeight_ConDiv = 0.0f;
	Training_DeltaWeight_SpaCod = 0.0f;
	Training_CumulativeExampleWeight_VisibleError = 0.0f;
	Training_CumulativeExampleWeight_ConDiv = 0.0f;
	Training_CumulativeExampleWeight_SpaCod = 0.0f;
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearTimers()
{
	TimerInference.Reset();
	TimerOther_ConDov.Reset();
	//TimerOther_SpaCod.Reset();
}

void nw_rbm::CRestrictedBoltzmannMachine::SetState_of_Units_Visible( const RBM_FLOAT* SrcPtrState )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].State = SrcPtrState[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetState_of_Units_Visible( const CSimpleTypeArray<RBM_FLOAT>& SrcListState )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].State = SrcListState.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetState_of_Units_Visible( RBM_FLOAT* DstPtrState ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstPtrState[i] = qU[i].State;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetState_of_Units_Visible( CSimpleTypeArray<RBM_FLOAT>& DstListState ) const
{
	DstListState.Resize(AmountUnit_Visible);
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstListState.pElement[i] = qU[i].State;
	return DstListState.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetExpectation_of_Units_Visible( const RBM_FLOAT* SrcPtrExpectation )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].Expectation = SrcPtrExpectation[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetExpectation_of_Units_Visible( const CSimpleTypeArray<RBM_FLOAT>& SrcListExpectation )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].Expectation = SrcListExpectation.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetExpectation_of_Units_Visible( RBM_FLOAT* DstPtrExpectation ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstPtrExpectation[i] = qU[i].Expectation;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetExpectation_of_Units_Visible( CSimpleTypeArray<RBM_FLOAT>& DstListExpectation ) const
{
	DstListExpectation.Resize(AmountUnit_Visible);
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstListExpectation.pElement[i] = qU[i].Expectation;
	return DstListExpectation.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetEnergy_of_Units_Visible( const RBM_FLOAT* SrcPtrEnergy )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].Energy = SrcPtrEnergy[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetEnergy_of_Units_Visible( const CSimpleTypeArray<RBM_FLOAT>& SrcListEnergy )
{
	int i;
	CUnitBasic* pU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		pU[i].Energy = SrcListEnergy.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetEnergy_of_Units_Visible( RBM_FLOAT* DstPtrEnergy ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstPtrEnergy[i] = qU[i].Energy;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetEnergy_of_Units_Visible( CSimpleTypeArray<RBM_FLOAT>& DstListEnergy ) const
{
	DstListEnergy.Resize(AmountUnit_Visible);
	int i;
	const CUnitBasic* qU = ListUnit_Visible.pElement;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		DstListEnergy.pElement[i] = qU[i].Energy;
	return DstListEnergy.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetState_of_Units_Hidden( const RBM_FLOAT* SrcPtrState )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].State = SrcPtrState[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetState_of_Units_Hidden( const CSimpleTypeArray<RBM_FLOAT>& SrcListState )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].State = SrcListState.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetState_of_Units_Hidden( RBM_FLOAT* DstPtrState ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstPtrState[i] = qU[i].State;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetState_of_Units_Hidden( CSimpleTypeArray<RBM_FLOAT>& DstListState ) const
{
	DstListState.Resize(AmountUnit_Hidden);
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstListState.pElement[i] = qU[i].State;
	return DstListState.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetExpectation_of_Units_Hidden( const RBM_FLOAT* SrcPtrExpectation )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].Expectation = SrcPtrExpectation[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetExpectation_of_Units_Hidden( const CSimpleTypeArray<RBM_FLOAT>& SrcListExpectation )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].Expectation = SrcListExpectation.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetExpectation_of_Units_Hidden( RBM_FLOAT* DstPtrExpectation ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstPtrExpectation[i] = qU[i].Expectation;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetExpectation_of_Units_Hidden( CSimpleTypeArray<RBM_FLOAT>& DstListExpectation ) const
{
	DstListExpectation.Resize(AmountUnit_Hidden);
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstListExpectation.pElement[i] = qU[i].Expectation;
	return DstListExpectation.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetEnergy_of_Units_Hidden( const RBM_FLOAT* SrcPtrEnergy )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].Energy = SrcPtrEnergy[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::SetEnergy_of_Units_Hidden( const CSimpleTypeArray<RBM_FLOAT>& SrcListEnergy )
{
	int i;
	CUnitBasic* pU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		pU[i].Energy = SrcListEnergy.pElement[i];
}

void nw_rbm::CRestrictedBoltzmannMachine::GetEnergy_of_Units_Hidden( RBM_FLOAT* DstPtrEnergy ) const
{
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstPtrEnergy[i] = qU[i].Energy;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::GetEnergy_of_Units_Hidden( CSimpleTypeArray<RBM_FLOAT>& DstListEnergy ) const
{
	DstListEnergy.Resize(AmountUnit_Hidden);
	int i;
	const CUnitBasic* qU = ListUnit_Hidden.pElement;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		DstListEnergy.pElement[i] = qU[i].Energy;
	return DstListEnergy.pElement;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetAmountWeightItem_Edge(int AmountWeightItem_Edge)
{
	this->AmountWeightItem_Edge = AmountWeightItem_Edge;
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Edge.Resize(AmountWeightItem_Edge);
	ListWeightItem_Edge.Fill(tWI);
}

void nw_rbm::CRestrictedBoltzmannMachine::SetAmountWeightItem_Visible(int AmountWeightItem_Visible)
{
	this->AmountWeightItem_Visible = AmountWeightItem_Visible;
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Visible.Resize(AmountWeightItem_Visible);
	ListWeightItem_Visible.Fill(tWI);
}

void nw_rbm::CRestrictedBoltzmannMachine::SetAmountWeightItem_Hidden(int AmountWeightItem_Hidden)
{
	this->AmountWeightItem_Hidden = AmountWeightItem_Hidden;
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Hidden.Resize(AmountWeightItem_Hidden);
	ListWeightItem_Hidden.Fill(tWI);
}

int nw_rbm::CRestrictedBoltzmannMachine::NewWeightItem_Edge()
{
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Edge.PushBack(tWI);
	return AmountWeightItem_Edge++;
}

int nw_rbm::CRestrictedBoltzmannMachine::NewWeightItem_Visible()
{
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Visible.PushBack(tWI);
	return AmountWeightItem_Visible++;
}

int nw_rbm::CRestrictedBoltzmannMachine::NewWeightItem_Hidden()
{
	CWeightItem tWI;
	tWI.ClearAll();
	ListWeightItem_Hidden.PushBack(tWI);
	return AmountWeightItem_Hidden++;
}

int nw_rbm::CRestrictedBoltzmannMachine::NewUnit_Visible(
	bool fIsRealValued, RBM_FLOAT Sigma, int WeightItemIndex_Visible )
{
	int ret = AmountUnit_Visible++;
	CUnitBasic tUnit;
	tUnit.myInit(
		fIsRealValued, Sigma, WeightItemIndex_Visible);
	ListUnit_Visible.PushBack(tUnit);
	return ret;
}

int nw_rbm::CRestrictedBoltzmannMachine::NewUnit_Hidden(
	bool fIsRealValued, RBM_FLOAT Sigma, int WeightItemIndex_Hidden )
{
	int ret = AmountUnit_Hidden++;
	CUnitBasic tUnit;
	tUnit.myInit(
		fIsRealValued, Sigma, WeightItemIndex_Hidden);
	ListUnit_Hidden.PushBack(tUnit);
	return ret;
}

void nw_rbm::CRestrictedBoltzmannMachine::NewRelation_Edge(
	int UnitIndex_Visible, int UnitIndex_Hidden, int WeightItemIndex_Edge )
{
	if (UnitIndex_Visible<0 || UnitIndex_Visible>=AmountUnit_Visible) return;
	if (UnitIndex_Hidden<0 || UnitIndex_Hidden>=AmountUnit_Hidden) return;
	if (WeightItemIndex_Edge<0 || WeightItemIndex_Edge>=AmountWeightItem_Edge) return;
	
	pConnectionManager_Visible->NewConnection(UnitIndex_Visible, UnitIndex_Hidden, WeightItemIndex_Edge);
	pConnectionManager_Hidden->NewConnection(UnitIndex_Hidden, UnitIndex_Visible, WeightItemIndex_Edge);
}

void nw_rbm::CRestrictedBoltzmannMachine::InitWeightItems()
{
	InitWeightItems(0.0f);
}

void nw_rbm::CRestrictedBoltzmannMachine::InitWeightItems( RBM_FLOAT Variance )
{
	int i, LWI;
	CWeightItem* PWI;

	SetAmountWeightItem_Edge(AmountWeightItem_Edge);
	SetAmountWeightItem_Visible(AmountWeightItem_Visible);
	SetAmountWeightItem_Hidden(AmountWeightItem_Hidden);

	if (Variance < MINIMUM_DOUBLE_EPS_C)
	{
		ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = 0.0f;
		ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = 0.0f;
		ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = 0.0f;
	}
	else
	{
		ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = (RBM_FLOAT)CMyRandom::Gaussian1D(0.0f, Variance);
		ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = (RBM_FLOAT)CMyRandom::Gaussian1D(0.0f, Variance);
		ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
		for ( i = 0; i < LWI; ++i )
			PWI[i].Value = (RBM_FLOAT)CMyRandom::Gaussian1D(0.0f, Variance);
	}
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearWeightItems_Value()
{
	int i, LWI;
	CWeightItem* PWI;
	ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].Value = 0.0f;
	ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].Value = 0.0f;
	ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].Value = 0.0f;
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearWeightItems_ConDiv()
{
	int i, LWI;
	CWeightItem* PWI;
	ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearConDiv();
	ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearConDiv();
	ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearConDiv();
}

void nw_rbm::CRestrictedBoltzmannMachine::ClearWeightItems_SpaCod()
{
	int i, LWI;
	CWeightItem* PWI;
	ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearSpaCod();
	ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearSpaCod();
	ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
	for ( i = 0; i < LWI; ++i )
		PWI[i].ClearSpaCod();
}

void nw_rbm::CRestrictedBoltzmannMachine::SaveWeightModelToFile( string strFN_WeightModel ) const
{
	ofstream outFile(strFN_WeightModel.c_str());
	SaveWeightModelToStream(outFile);
	outFile.close();
	outFile.clear();
}

bool nw_rbm::CRestrictedBoltzmannMachine::LoadWeightModelFromFile( string strFN_WeightModel )
{
	ifstream inFile(strFN_WeightModel.c_str());
	bool ret = LoadWeightModelFromStream(inFile);
	inFile.close();
	inFile.clear();
	return ret;
}

void nw_rbm::CRestrictedBoltzmannMachine::SaveWeightModelToStream( ostream& outStream ) const
{
	int i, t, LWI;
	const CWeightItem* PWI;

	outStream.unsetf(ios::showpos);
	ListWeightItem_Visible.GetPvLenInfo(PWI, LWI);
	outStream<<endl<<LWI<<endl<<endl;
	for ( i = 0; i < LWI; ++i )
	{
		outStream.precision(11);
		outStream.setf(ios::showpos);
		outStream.setf(ios::scientific);
		outStream<<"  "<<PWI[i].Value;
		if (i==LWI-1) { outStream<<endl; continue; }
	}

	outStream.unsetf(ios::showpos);
	ListWeightItem_Hidden.GetPvLenInfo(PWI, LWI);
	outStream<<endl<<LWI<<endl<<endl;
	for ( i = 0; i < LWI; ++i )
	{
		outStream.precision(11);
		outStream.setf(ios::showpos);
		outStream.setf(ios::scientific);
		outStream<<"  "<<PWI[i].Value;
		if (i==LWI-1) { outStream<<endl; continue; }
	}

	outStream.unsetf(ios::showpos);
	ListWeightItem_Edge.GetPvLenInfo(PWI, LWI);
	outStream<<endl<<LWI<<endl<<endl;
	for ( i = 0; i < LWI; ++i )
	{
		outStream.precision(11);
		outStream.setf(ios::showpos);
		outStream.setf(ios::scientific);
		outStream<<"  "<<PWI[i].Value;
		if ((PrintEdgeWeightAmountPerLine>1) && ((i+1)%PrintEdgeWeightAmountPerLine == 0))
		{
			outStream<<endl;
			t = (i+1)/PrintEdgeWeightAmountPerLine;
			if ((PrintEdgeWeightAmountPerKind>1) && ((t+1)%PrintEdgeWeightAmountPerKind == 0))
				outStream<<endl;
			continue;
		}
		if (i==LWI-1) { outStream<<endl; continue; }
	}

	outStream.unsetf(ios::showpos);

	outStream<<endl;
}

bool nw_rbm::CRestrictedBoltzmannMachine::LoadWeightModelFromStream( istream& inStream )
{
	int i;
	int* PLen;
	CList_WeightItem* PList;

	PLen = &AmountWeightItem_Visible;
	PList = &ListWeightItem_Visible;
	if (!(inStream>>*PLen)) return false;
	PList->Resize(*PLen);
	for ( i = 0; i < *PLen; ++i )
	{
		PList->pElement[i].ClearAll();
		if (!(inStream>>PList->pElement[i].Value)) return false;
	}

	PLen = &AmountWeightItem_Hidden;
	PList = &ListWeightItem_Hidden;
	if (!(inStream>>*PLen)) return false;
	PList->Resize(*PLen);
	for ( i = 0; i < *PLen; ++i )
	{
		PList->pElement[i].ClearAll();
		if (!(inStream>>PList->pElement[i].Value)) return false;
	}

	PLen = &AmountWeightItem_Edge;
	PList = &ListWeightItem_Edge;
	if (!(inStream>>*PLen)) return false;
	PList->Resize(*PLen);
	for ( i = 0; i < *PLen; ++i )
	{
		PList->pElement[i].ClearAll();
		if (!(inStream>>PList->pElement[i].Value)) return false;
	}

	return true;
}

void nw_rbm::CRestrictedBoltzmannMachine::InferenceEnergy_of_Units_Visible(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper )
{
#if COUNT_TIME
	TimerInference.Start();
#endif
	if (p_RBM_InferenceHelper == NULL )
	{
		C_RBM_InferenceHelper tHelper;
		tHelper.InferenceEnergy_of_Units_Visible(this);
	}
	else
		p_RBM_InferenceHelper->InferenceEnergy_of_Units_Visible(this);
#if COUNT_TIME
	TimerInference.Stop();
#endif
}

void nw_rbm::CRestrictedBoltzmannMachine::InferenceEnergy_of_Units_Hidden(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper )
{
#if COUNT_TIME
	TimerInference.Start();
#endif
	if (p_RBM_InferenceHelper == NULL )
	{
		C_RBM_InferenceHelper tHelper;
		tHelper.InferenceEnergy_of_Units_Hidden(this);
	}
	else
		p_RBM_InferenceHelper->InferenceEnergy_of_Units_Hidden(this);
#if COUNT_TIME
	TimerInference.Stop();
#endif
}

void nw_rbm::CRestrictedBoltzmannMachine::ComputeExpectation_of_Units_Visible()
{
	int i;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		ListUnit_Visible.pElement[i].ComputeExpectation();
}

void nw_rbm::CRestrictedBoltzmannMachine::ComputeExpectation_of_Units_Hidden()
{
	int i;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		ListUnit_Hidden.pElement[i].ComputeExpectation();
}

void nw_rbm::CRestrictedBoltzmannMachine::SampleState_of_Units_Visible()
{
	int i;
	for ( i = 0; i < AmountUnit_Visible; ++i )
		ListUnit_Visible.pElement[i].SampleState();
}

void nw_rbm::CRestrictedBoltzmannMachine::SampleState_of_Units_Hidden()
{
	int i;
	for ( i = 0; i < AmountUnit_Hidden; ++i )
		ListUnit_Hidden.pElement[i].SampleState();
}

void nw_rbm::CRestrictedBoltzmannMachine::FullInferenceFlow_Visible(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper )
{
	InferenceEnergy_of_Units_Visible(p_RBM_InferenceHelper);
	ComputeExpectation_of_Units_Visible();
	SampleState_of_Units_Visible();
}

void nw_rbm::CRestrictedBoltzmannMachine::FullInferenceFlow_Hidden(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper )
{
	InferenceEnergy_of_Units_Hidden(p_RBM_InferenceHelper);
	ComputeExpectation_of_Units_Hidden();
	SampleState_of_Units_Hidden();
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::GetUnitBiasValue_Visible( int UnitIndex_Visible ) const
{
	if (UnitIndex_Visible<0 || UnitIndex_Visible>=AmountUnit_Visible) return 0.0f;
	const CUnitBasic& tUV = ListUnit_Visible.pElement[UnitIndex_Visible];
	if (tUV.BiasIndex >= 0)
		return ListWeightItem_Visible.pElement[tUV.BiasIndex].Value;
	else
		return 0.0f;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::GetUnitBiasValue_Hidden( int UnitIndex_Hidden ) const
{
	if (UnitIndex_Hidden<0 || UnitIndex_Hidden>=AmountUnit_Hidden) return 0.0f;
	const CUnitBasic& tUH = ListUnit_Hidden.pElement[UnitIndex_Hidden];
	if (tUH.BiasIndex >= 0)
		return ListWeightItem_Hidden.pElement[tUH.BiasIndex].Value;
	else
		return 0.0f;
}

void nw_rbm::CRestrictedBoltzmannMachine::SetBiasOnlyEnergy_of_Units_Visible()
{
	int i_V;
	CUnitBasic* pUV = ListUnit_Visible.pElement;
	const CWeightItem* qWI = ListWeightItem_Visible.pElement;
	for ( i_V = 0; i_V < AmountUnit_Visible; ++i_V, ++pUV )
	{
		if (pUV->BiasIndex >= 0)
			pUV->Energy = qWI[pUV->BiasIndex].Value;
		else
			pUV->Energy = 0.0;
	}
}

void nw_rbm::CRestrictedBoltzmannMachine::SetBiasOnlyEnergy_of_Units_Hidden()
{
	int i_H;
	CUnitBasic* pUH = ListUnit_Hidden.pElement;
	const CWeightItem* qWI = ListWeightItem_Hidden.pElement;
	for ( i_H = 0; i_H < AmountUnit_Hidden; ++i_H, ++pUH )
	{
		if (pUH->BiasIndex >= 0)
			pUH->Energy = qWI[pUH->BiasIndex].Value;
		else
			pUH->Energy = 0.0;
	}
}

void nw_rbm::CRestrictedBoltzmannMachine::CopyUnitsExpectationToState_Visible()
{
	int i;
	for ( i = 0; i < AmountUnit_Visible; ++i )
	{
		CUnitBasic& tUnit_Visible = ListUnit_Visible.pElement[i];
		tUnit_Visible.State = tUnit_Visible.Expectation;
	}
}

void nw_rbm::CRestrictedBoltzmannMachine::CopyUnitsExpectationToState_Hidden()
{
	int j;
	for ( j = 0; j < AmountUnit_Hidden; ++j )
	{
		CUnitBasic& tUnit_Hidden = ListUnit_Hidden.pElement[j];
		tUnit_Hidden.State = tUnit_Hidden.Expectation;
	}
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::ComputeGradient_ConDiv_GivenUnitsVisible(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper,
	bool fAlreadyInferencedHidden,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment,
	int AmountInferenceIteration )
{
	int i;
	CList_Value Vs, Hs, PHs;
	CList_Value Vt, Ht, PHt;
	const RBM_FLOAT *qVs, *qHs, *qPHs;
	const RBM_FLOAT *qVt, *qHt, *qPHt;

	//CWeightItem *qEW = ListWeightItem_Edge.pElement;
	CWeightItem *qVC = ListWeightItem_Visible.pElement;
	CWeightItem *qHC = ListWeightItem_Hidden.pElement;

	if (!fAlreadyInferencedHidden)
		this->FullInferenceFlow_Hidden(p_RBM_InferenceHelper);
	qVs = this->GetState_of_Units_Visible(Vs);
	qHs = this->GetState_of_Units_Hidden(Hs);
	qPHs = this->GetExpectation_of_Units_Hidden(PHs);
	for ( i = 1; i <= AmountInferenceIteration; ++i )
	{
		this->FullInferenceFlow_Visible(p_RBM_InferenceHelper);
		this->FullInferenceFlow_Hidden(p_RBM_InferenceHelper);
	}
	qVt = this->GetState_of_Units_Visible(Vt);
	qHt = this->GetState_of_Units_Hidden(Ht);
	qPHt = this->GetExpectation_of_Units_Hidden(PHt);
	this->SetState_of_Units_Visible(Vs);
	this->SetState_of_Units_Hidden(PHs);

#if COUNT_TIME
	TimerOther_ConDov.Start();
#endif
	RBM_FLOAT VisibleError = 
		p_RBM_InferenceHelper->ComputeEdgeGradient_ConDiv(
		this, qVs, qPHs, qVt, qPHt, ExampleWeight, GradientAmendment);
#if COUNT_TIME
	TimerOther_ConDov.Stop();
#endif

	Training_CumulativeVisibleError += VisibleError;
	Training_CumulativeExampleWeight_VisibleError += ExampleWeight;
	Training_CumulativeExampleWeight_ConDiv += ExampleWeight;
	return VisibleError;
}

const RBM_FLOAT* nw_rbm::CRestrictedBoltzmannMachine::MakeHiddenDerivative_for_SparseCoding(
	CSimpleTypeArray<RBM_FLOAT>& vHiddenDerivative )
{
	int j;
	vHiddenDerivative.Resize(AmountUnit_Hidden);
	RBM_FLOAT* ret = vHiddenDerivative.pElement;
	for ( j = 0; j < AmountUnit_Hidden; ++j )
	{
		CUnitBasic& tHj = ListUnit_Hidden.pElement[j];
		ret[j] = 2.0f * tHj.Expectation * ((RBM_FLOAT)1.0f - tHj.Expectation);
	}
	return ret;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::ComputeGradient_SpaCod_GivenUnitsVisible(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper,
	RBM_FLOAT SparseLevel,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment )
{
	RBM_FLOAT HiddenPenalty = 0.0f;

	int ee, i, j, o;
	RBM_FLOAT dw;
	CList_Value Vs;
	const RBM_FLOAT *qVs;
	const int *qUI, *qWI;

	CWeightItem *qEW = ListWeightItem_Edge.pElement;
	CWeightItem *qVC = ListWeightItem_Visible.pElement;
	CWeightItem *qHC = ListWeightItem_Hidden.pElement;

	this->FullInferenceFlow_Hidden(p_RBM_InferenceHelper);
	qVs = this->GetState_of_Units_Visible(Vs);
	ee = i = j = o = 0;
	qUI = qWI = NULL;

	GradientAmendment *= ExampleWeight;

	CSimpleTypeArray<RBM_FLOAT> vHiddenDerivative;

	RBM_FLOAT tx, ty;
	const RBM_FLOAT* qHD;

	// Tianqi Chen
	qHD = this->MakeHiddenDerivative_for_SparseCoding(vHiddenDerivative);
	//qHD = CRestrictedBoltzmannMachine::MakeHiddenDerivative_for_SparseCoding(vHiddenDerivative);

	for ( j = 0; j < AmountUnit_Hidden; ++j )
	{
		CUnitBasic& tHj = ListUnit_Hidden.pElement[j];
		tx = SparseLevel - tHj.Expectation;
		ty = qHD[j];

		o = tHj.BiasIndex;
		dw = ty * 1.0f;
		if (o >= 0)
		{
			qHC[o].SpaCod_CumulativeError += tx * ExampleWeight;
			qHC[o].SpaCod_CumulativeDerivative += dw * GradientAmendment;
			qHC[o].SpaCod_CumulativeWeight += ExampleWeight;
		}
	}

	HiddenPenalty = 0.0f;
	for ( o = 0; o < AmountWeightItem_Hidden; ++o )
	{
		tx = qHC[o].SpaCod_CumulativeError;
		ty = qHC[o].SpaCod_CumulativeWeight;
		dw = (abs(ty)<MINIMUM_DOUBLE_EPS_C ? 0.0f : tx/ty);
		HiddenPenalty += dw * dw;
	}
	if (AmountWeightItem_Hidden > MINIMUM_DOUBLE_EPS_C)
		HiddenPenalty = sqrt(HiddenPenalty / AmountWeightItem_Hidden);

	Training_CumulativeExampleWeight_SpaCod += ExampleWeight;
	Training_CumulativeHiddenPenalty = HiddenPenalty;
	return HiddenPenalty;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::Update_ContrastiveDivergence(
	RBM_FLOAT LearningRate, RBM_FLOAT Decay_Mask, RBM_FLOAT Decay_Bias, RBM_FLOAT Momentum )
{
	RBM_FLOAT ret = 0.0f;
	int ddd = 0;
	int o;
	RBM_FLOAT t;
	CWeightItem *qEW, *qVC, *qHC;

	if (abs(Training_CumulativeExampleWeight_ConDiv) < MINIMUM_DOUBLE_EPS_C)
		Training_CumulativeExampleWeight_ConDiv = 1.0;

	qEW = ListWeightItem_Edge.pElement;
	qVC = ListWeightItem_Visible.pElement;
	qHC = ListWeightItem_Hidden.pElement;
	for ( o = 0; o < AmountWeightItem_Edge; ++o )
	{
		CWeightItem& tWI = qEW[o];
		if (!tWI.fValueFixed)
		{
			t = tWI.ConDiv_NextIncrease;
#if FLAG_AVERAGE_WEIGHT_GRADIENT
			t /= (abs(tWI.ConDiv_CumulativeWeight)<MINIMUM_DOUBLE_EPS_C ? 1.0f : tWI.ConDiv_CumulativeWeight);
#else
			t /= Training_CumulativeExampleWeight_ConDiv;
#endif
			ret += abs(t); ++ddd;
			t = (t - Decay_Mask * tWI.Value) * LearningRate + tWI.ConDiv_LastIncrease * Momentum;
		}
		else
			t = 0.0f;
		tWI.Value += t;
		tWI.ConDiv_LastIncrease = t;
		tWI.ConDiv_NextIncrease = 0.0f;
		tWI.ConDiv_CumulativeWeight = 0.0f;
	}
	for ( o = 0; o < AmountWeightItem_Visible; ++o )
	{
		CWeightItem& tWI = qVC[o];
		if (!tWI.fValueFixed)
		{
			t = tWI.ConDiv_NextIncrease;
#if FLAG_AVERAGE_WEIGHT_GRADIENT
			t /= (abs(tWI.ConDiv_CumulativeWeight)<MINIMUM_DOUBLE_EPS_C ? 1.0f : tWI.ConDiv_CumulativeWeight);
#else
			t /= Training_CumulativeExampleWeight_ConDiv;
#endif
			ret += abs(t); ++ddd;
			t = (t - Decay_Bias * tWI.Value) * LearningRate + tWI.ConDiv_LastIncrease * Momentum;
		}
		else
			t = 0.0f;
		tWI.Value += t;
		tWI.ConDiv_LastIncrease = t;
		tWI.ConDiv_NextIncrease = 0.0f;
		tWI.ConDiv_CumulativeWeight = 0.0f;
	}
	for ( o = 0; o < AmountWeightItem_Hidden; ++o )
	{
		CWeightItem& tWI = qHC[o];
		if (!tWI.fValueFixed)
		{
			t = tWI.ConDiv_NextIncrease;
#if FLAG_AVERAGE_WEIGHT_GRADIENT
			t /= (abs(tWI.ConDiv_CumulativeWeight)<MINIMUM_DOUBLE_EPS_C ? 1.0f : tWI.ConDiv_CumulativeWeight);
#else
			t /= Training_CumulativeExampleWeight_ConDiv;
#endif
			ret += abs(t); ++ddd;
			t = (t - Decay_Bias * tWI.Value) * LearningRate + tWI.ConDiv_LastIncrease * Momentum;
		}
		else
			t = 0.0f;
		tWI.Value += t;
		tWI.ConDiv_LastIncrease = t;
		tWI.ConDiv_NextIncrease = 0.0f;
		tWI.ConDiv_CumulativeWeight = 0.0f;
	}

	if (ddd) ret /= ddd;
	Training_DeltaWeight_ConDiv += ret;
	Training_CumulativeExampleWeight_ConDiv = 0.0f;
	return ret;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::Update_SparseCoding(
	RBM_FLOAT LearningRate, RBM_FLOAT SparseCodingLambda )
{
	RBM_FLOAT ret = 0.0f;
	int ddd = 0;

	int o;
	RBM_FLOAT t, tx, ty, ls;

	ls = LearningRate * SparseCodingLambda;

	if (abs(Training_CumulativeExampleWeight_SpaCod) < MINIMUM_DOUBLE_EPS_C)
		Training_CumulativeExampleWeight_SpaCod = 1.0;

	CWeightItem *qHC = ListWeightItem_Hidden.pElement;

	for ( o = 0; o < AmountWeightItem_Hidden; ++o )
	{
		CWeightItem& tWI = qHC[o];
		if (!tWI.fValueFixed)
		{
			if (abs(tWI.SpaCod_CumulativeWeight) < MINIMUM_DOUBLE_EPS_C)
				tx = ty = 0.0f;
			else
			{
				tx = tWI.SpaCod_CumulativeError / tWI.SpaCod_CumulativeWeight;
				ty = tWI.SpaCod_CumulativeDerivative / tWI.SpaCod_CumulativeWeight;
			}
			t = ls * tx * ty;
#if FLAG_AVERAGE_WEIGHT_GRADIENT
#else
			t *= tWI.SpaCod_CumulativeWeight / Training_CumulativeExampleWeight_SpaCod;
#endif
			ret += abs(t); ++ddd;
		}
		else
			t = 0.0f;
		tWI.Value += t;
		tWI.SpaCod_CumulativeDerivative = 0.0f;
		tWI.SpaCod_CumulativeError = 0.0f;
		tWI.SpaCod_CumulativeWeight = 0.0f;
	}
	if (ddd) ret /= ddd;
	ret /= LearningRate;
	Training_CumulativeExampleWeight_SpaCod = 0.0f;
	Training_DeltaWeight_SpaCod += ret;
	return ret;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::GetCumulativeVisibleError() const
{
	if (abs(Training_CumulativeExampleWeight_VisibleError) > MINIMUM_DOUBLE_EPS_C)
		return sqrt(Training_CumulativeVisibleError / Training_CumulativeExampleWeight_VisibleError);
	else
		return 0.0f;
}

RBM_FLOAT nw_rbm::CRestrictedBoltzmannMachine::GetCumulativeHiddenPenalty() const
{
	return Training_CumulativeHiddenPenalty;
}

void nw_rbm::CRestrictedBoltzmannMachine::ComputeBothGradient(
	C_RBM_InferenceHelper* p_RBM_InferenceHelper,
	RBM_FLOAT SparseLevel,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment,
	int AmountInferenceIteration )
{
	ComputeGradient_SpaCod_GivenUnitsVisible(
		p_RBM_InferenceHelper,
		SparseLevel,
		ExampleWeight, GradientAmendment);
	ComputeGradient_ConDiv_GivenUnitsVisible(
		p_RBM_InferenceHelper,
		true,
		ExampleWeight, GradientAmendment,
		AmountInferenceIteration);
}

void nw_rbm::CRestrictedBoltzmannMachine::UpdateBothGradient(
	RBM_FLOAT LearningRate,
	RBM_FLOAT SpaCod_Lambda,
	RBM_FLOAT ConDiv_Decay_Mask, RBM_FLOAT ConDiv_Decay_Bias, RBM_FLOAT ConDiv_Momentum )
{
	Update_SparseCoding(LearningRate, SpaCod_Lambda);
	Update_ContrastiveDivergence(LearningRate, ConDiv_Decay_Mask, ConDiv_Decay_Bias, ConDiv_Momentum);
}

